In [1]:
%%HTML
<style type="text/css">
table.dataframe td, table.datafreame th {
border: 1px black solid;
color: black;
}
In [5]:
df.shape
Out[5]:
In [6]:
# get the basic information of the dataset
df.info()
In [7]:
#figure if there some null value will effect on our project
df.isnull().sum()
Out[7]:
In [8]:
#filter and left the useful information only
df.drop(columns = ['tweet_id', 'name'], axis = 1, inplace = True)
In [9]:
df.columns
Out[9]:
Finding the sentiment for each tweets¶
In [11]:
from nltk.sentiment.vader import SentimentIntensityAnalyzer
In [12]:
sid = SentimentIntensityAnalyzer()
In [13]:
sid.polarity_scores(df.text[0])
Out[13]:
In [14]:
df['polarity_score'] = df.text.apply(lambda x: sid.polarity_scores(x))
In [15]:
df.head()
Out[15]:
In [16]:
df['score'] = df.polarity_score.apply(lambda x: x['compound'])
In [17]:
def find_sentiment(x):
if x > 0:
return "Positive"
elif x < 0:
return "Negative"
else:
return "Neutral"
In [18]:
df["sentiment"] = df.score.apply(lambda x: find_sentiment(x))
In [19]:
df.head()
Out[19]:
now we can drop polaarity score and score column. Also visulize the result of our figure¶
In [20]:
df.drop(columns = ['polarity_score', 'score'], axis = 1, inplace = True)
In [21]:
df.head()
Out[21]:
In [22]:
df.sentiment.value_counts()
Out[22]:
In [23]:
sns.countplot(x = 'sentiment', data = df)
Out[23]:
In [24]:
fig = go.Figure()
c = ['green','red','blue']
fig.add_trace(go.Bar(x = df.sentiment.value_counts().index,
y = df.sentiment.value_counts(),
marker_color = c,
text = df.sentiment.value_counts(),
textposition='auto'))
fig.update_layout(title = dict(text = "Sentiment count"))
Topic Modeling of the tweets¶
In this section, according to key-words. Texts will fit into a related topic.
In [25]:
import spacy
In [26]:
nlp = spacy.load('en_core_web_sm')
In [27]:
def tokenization(s):
text = nlp(s.lower())
token = []
for t in text:
token.append(t.lemma_)
return token
In [28]:
df["lemma"] = df.text.apply(lambda x: tokenization(x))
In [29]:
df.head()
Out[29]:
In [30]:
def stopwords(s):
l = []
for i in s:
if i not in nlp.Defaults.stop_words:
l.append(i)
return " ".join(l)
In [31]:
df['proccesed_text'] = df.lemma.apply(lambda x: stopwords(x))
In [32]:
from sklearn.feature_extraction.text import TfidfVectorizer
In [33]:
tdf = TfidfVectorizer(max_df=0.9, min_df=10)
In [34]:
dtm = tdf.fit_transform(df["proccesed_text"])
In [35]:
#LDA is applied in this part for machine learning
from sklearn.decomposition import ss
In [36]:
lda = LatentDirichletAllocation(n_components= 10, random_state= 0)
In [37]:
lda.fit(dtm)
Out[37]:
Shown the result of the top 20 words for each topics.¶
In [38]:
for i, topic in enumerate(lda.components_):
print(f"Top 20 words for topic {i}")
print([tdf.get_feature_names()[index] for index in topic.argsort()[-20:]])
print("\n")
In [39]:
topic = lda.transform(dtm)
In [40]:
df["topic"] = topic.argmax(axis = 1)
In [41]:
df.head()
Out[41]:
In [42]:
df.drop(columns=['lemma', 'proccesed_text'], axis =1, inplace = True)
In [43]:
df.head()
Out[43]:
In [44]:
reason = df.negativereason.value_counts()
In [45]:
fig = go.Figure()
fig.add_trace(go.Bar(x = reason.index,
y = reason))
In [46]:
df.retweet_count.unique()
Out[46]:
Word cloud¶
In order to anaylze compaint, we only use tweets with negatvie attitude according to the sentiment anaylze
In [47]:
df_neg = df[df['sentiment'] == 'Negative']
df_neg.head()
Out[47]:
In [48]:
df_neg.to_csv (r'negativeTweets.csv', index = False, header=True)
In [49]:
import collections
import numpy as np
import pandas as pd
import matplotlib.cm as cm
import matplotlib.pyplot as plt
from matplotlib import rcParams
from wordcloud import WordCloud, STOPWORDS
%matplotlib inline
from nltk.tokenize.treebank import TreebankWordDetokenizer
from nltk import word_tokenize
In [50]:
# read the tweet from the data.
def removeAirline (userTyped):
tokens = word_tokenize(userTyped)
res = ''
if tokens[0] == '@':
tokens.pop(0)
tokens.pop(0)
res = TreebankWordDetokenizer().detokenize(tokens)
else:
res = userTyped
return res
In [51]:
# Collect the key-words from a certain reason.
# Each reason will having it's own word Cloud as output.
def wordCloud_res(reason):
df_res = df_neg[df_neg['negativereason'] == reason]
all_headlines = ' '
for line in df_res['text'].str.lower():
all_headlines += removeAirline(line)
stopwords = STOPWORDS
stopwords.add('will')
wordcloud = WordCloud(stopwords=stopwords, background_color="white", max_words=500).generate(all_headlines)
print('With the reason of: ' + reason)
rcParams['figure.figsize'] = 10, 20
plt.imshow(wordcloud)
plt.axis("off")
plt.show()
In [52]:
reason = df_neg['negativereason'].unique()
reason
Out[52]:
Most reason of negative sentiment is about:¶
'Bad Flight', 'Late Flight','Customer Service Issue', 'Flight Booking Problems','Lost Luggage', 'Flight Attendant Complaints', 'Cancelled Flight', 'Damaged Luggage', 'longlines'
These will be the key words for the robot to analyze¶
In the following steps, all the word clouds for each reason is shown
In [53]:
wordCloud_res(reason[0])
In [54]:
wordCloud_res(reason[3])
In [55]:
wordCloud_res(reason[4])
In [56]:
wordCloud_res(reason[5])
In [57]:
wordCloud_res(reason[6])
In [58]:
wordCloud_res(reason[7])
In [59]:
wordCloud_res(reason[8])
In [60]:
wordCloud_res(reason[9])
In [61]:
wordCloud_res(reason[10])
